# -*- coding: utf-8 -*-
# @Time    : 2018/11/29 15:20
import os
import sys

path = os.path.abspath(os.path.dirname(os.getcwd()))
sys.path.append(path)
import re
import log,threading
from time import sleep
from filter import config
from collections import OrderedDict
from multiprocessing import Process, Queue, Pool
#isalpha 都是语言或字母
db_million = config.db_million
url_0 = db_million['url_0']
url_1 = db_million['url_1']
url_2 = db_million['url_2']
url_3 = db_million['url_3']
url_4 = db_million['url_4']
url_5 = db_million['url_5']
url_6 = db_million['url_6']
url_7 = db_million['url_7']
url_8 = db_million['url_8']
url_9 = db_million['url_9']
url_a = db_million['url_a']
url_b = db_million['url_b']
url_c = db_million['url_c']
url_d = db_million['url_d']
url_e = db_million['url_e']
url_f = db_million['url_f']
url_xiaoyuzhong = config.url_xiaoyuzhong
url_path = config.url_path
url_s = {"0":url_0, "1":url_1, "2":url_2, "3":url_3, "4":url_4, "5":url_5, "6":url_6, "7":url_7, "8":url_8,
         "9":url_9, "a":url_a, "b":url_b, "c":url_c, "d":url_d, "e":url_e, "f":url_f}
minganci = config.minganci
n = 0
new_log = log.Log("yuyanshibie")
class YuYan(object):
    def __init__(self):
        self.charu_num = 0
        pass

    def find_yuan_two(self, url_md5):
        return url_s["%s" % (url_md5[0])].find_one({"md5": url_md5})

    def is_chinese(self,uchar):
        if uchar >= u'\u4E00' and uchar <= u'\u9fff':
            return True
        else:
            return False

    def is_japan(self,uchar):
        if uchar >= u'\u0800' and uchar <= u'\u4e00':
            return True
        else:
            return False

    def is_english(self,uchar):
        if ( uchar >= u'\u0041' and uchar <= u'\u005A' ) or ( uchar >= u'\u0061' and uchar <= u'\u007A'):
            return True
        else:
            return False

    def find_str(self, zifuchuan):
        new_zifu = zifuchuan[:int(len(zifuchuan)*0.5)]
        if not new_zifu:
            return True
        china_num = 0
        english_num = 0
        th_num = 0
        for i in new_zifu:
            # 除字符
            if i.isalpha():
                if self.is_english(i):
                    english_num += 1
                elif self.is_chinese(i):
                    china_num += 1
                else:
                    th_num =1
                    break
        if th_num:
            return False
            # print("小语种")
        elif english_num>china_num:
            return True
            # print("英文")
        elif china_num>=english_num:
            return True
            # print("中文")
        else:
            return False
            # print("判断不出")

    def find_yuan(self,path):
        try:
            url_md5 = (str(path).split("/")[-1]).split(".")[0]
            if url_md5:
                try:
                    yuan = self.find_yuan_two(url_md5)
                    if not yuan:
                        return
                except:
                    print("not find md5 %s"%(url_md5))
                    new_log.error("not url_md5: %s" % path)
                    return
                try:
                    source = self.find_str(yuan['title'])
                    if source:
                        nr = yuan['summary']
                        source = self.find_str(nr)
                        if source:
                            self.insert_mongo(source, url_md5, path)
                        #     #敏感词识别
                        #     for mingan in minganci:
                        #         if nr.find(mingan)>-1:
                        #             new_log.info("find minganci : %s : %s" % (url_md5, mingan))
                        #             source = False
                        #             break
                    # print(source)
                except Exception as e:
                    sleep(1)
                    new_log.error("this is not summary and title path: %s error: %s "%(path,e))
                    print("this is not summary and title path: %s error: %s"%(path,e))
        except:
            new_log.error("not url_md5: %s" % path)

    def insert_mongo(self,souce,md5,path):
        try:
            save = OrderedDict()
            save['md5'] = md5
            save['path'] = path
            save['state'] = 0
            if souce:
                url_path.insert(save)
            else:
                url_xiaoyuzhong.insert(save)
        except Exception as e:
            if str(e).find("E11000") > -1:
                return
            print("插入失败：%s" % (e))


    def delete_q(self,q):
        print("stare : %s " % (os.getpid()))
        A = 0
        while True:
            try:
                # return
                if q.qsize() <= 0:
                    if A == 5:
                        print("%s break" % (os.getpid()))
                        return
                    A += 1
                    print("%s sleep(2)"%(os.getpid()))
                    sleep(2)
                    continue
                A = 0
                item = q.get()
                self.charu_num += 1
                if self.charu_num%1000 == 0:
                    print("%s:  %s"%(os.getpid(),self.charu_num))
                # 寻找元数据
                self.find_yuan(item)
            except Exception as e:
                print(e)


    def read(self, path, q):
        global n
        # 返回工作目录下的文件
        try:
            files = os.listdir(path)
        except:
            return
        files_num = len(files)
        if files_num == 0:
            return
        # 判断第一个目录是不是文件
        if (os.path.isfile("%s/%s" % (path, files[0]))):
            print("path:%s  files_num:%s" % (path, files_num))
            num = 0
            for i in files:
                num += 1
                if num % 1000 == 0:
                    print("paht:%s  files_num:%s" % (path, num))
                if (i.split(".")[0]).isalnum():
                    q.put("%s/%s" % (path, i))
                    n += 1
            return
        # 可以操作的文件
        else:
            for i in files:
                new_path = "%s/%s"%(path,i)
                #print(new_path)
                self.read(new_path, q)
        print('生产者P 0; 共生产 {} 条数据'.format(n))




if __name__ == '__main__':
    yy = YuYan()
    new_path = config.file_path
    q = Queue(maxsize=20000)
    p1 = Process(target=yy.read, args=(new_path,q,))
    p1.start()
    trader = []
    for i in range(10):
        pr = Process(target=yy.delete_q, args=(q,))
        pr.start()
        trader.append(pr)

    for i in trader:
        i.join()
    print('this is %s' % (os.getpid()))
    p1.join()
    # path = "20181121/0/02/00278c41d3156943b24501fe348f5279.doc"
    # yy.find_yuan(path)

